from scrapy import Request, Spider
from getContent import get_content


class ContentSpider(Spider):
    name = "content"

    def start_requests(self):
        with open("urls.txt", encoding="utf-8") as f:
            items = f.read().split('\n')
        for item in items:
            id = item.split('\t')[0]
            url = item.split('\t')[1]
            yield Request(url=url, callback=self.parse, meta={'id': id})

    def parse(self, response):
        print(response.url)
        title, content = get_content(response.text, response.url)
        yield {'id': response.meta['id'], 'content': content}
